﻿using ChoWorkflowServer.Model;
using Cinchoo.Core;
using Cinchoo.Core.Diagnostics;
using Cinchoo.Core.ETL;
using Cinchoo.Core.IO;
using Cinchoo.Core.Net.Sockets;
using Cinchoo.Core.Services;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace ChoWorkflowServer
{
    public partial class ChoWorkflowServerEx
    {
        private void ProcessTaskFeedbackStatusCommand(ChoTaskFeedbackStatusCommand cmd)
        {
            if (cmd == null) return;
            ChoETLFeedbackService.DataModel.AddOrUpdateTaskBatch(cmd.BatchId, cmd.TaskName, cmd.ScheduleName, cmd.Status, cmd.CmdParameters, null,
                cmd.LogMsg, cmd.ProcessStart, cmd.ProcessEnd, cmd.Duration, null, cmd.ExitCode,
                cmd.RetryCount, cmd.ModifiedBy, Environment.MachineName);
        }

        [ChoStreamProfile("Commands Received...", Name = "wfCommands", OuterProfileName = null)]
        internal void OnCommandMsgObjectReceived(ChoConnectionState state, ChoCommandMessage msg)
        {
            if (msg == null) return;

            if (msg.Command is ChoTaskActionCommand)
            {
                ChoProfile.WriteLine(ChoObject.ToString(msg));
                Task.Factory.StartNew(ProcessTaskActionCommand, (object)new Tuple<object, ChoConnectionState>(msg, state)).LogExceptions();
            }
            else if (msg.Command is ChoTaskFeedbackStatusCommand)
            {
                ChoProfile.WriteLine(ChoObject.ToString(msg));
                ChoETLFeedbackService.Queue.Enqueue<ChoTaskFeedbackStatusCommand>(ProcessTaskFeedbackStatusCommand, msg.Command as ChoTaskFeedbackStatusCommand);
            }
            else
            {
                if (!(msg.Command is IChoNoTrackingCommand))
                    ChoProfile.WriteLine(ChoObject.ToString(msg));
                Task.Factory.StartNew(ProcessClientOtherCommand, new Tuple<ChoConnectionState, ChoCommandMessage>(state, msg)).LogExceptions();
            }
        }

        private void ProcessClientOtherCommand(object state)
        {
            Tuple<ChoConnectionState, ChoCommandMessage> tuple = state as Tuple<ChoConnectionState, ChoCommandMessage>;
            if (tuple == null) return;

            ChoConnectionState connState = tuple.Item1 as ChoConnectionState;
            ChoCommandMessage payload = tuple.Item2 as ChoCommandMessage;

            try
            {
                if (payload == null || payload.Command == null) return;

                if (payload.Command is ChoTaskMgmtCommand)
                {
                    ProcessTaskMgmtCommand(payload.Command as ChoTaskMgmtCommand, payload.TimeStamp, payload.User, payload.OriginHostName);
                    //Republish command to other clients
                    //if (_connectionManager != null)
                    //    _connectionManager.BroadcastCommand(payload);
                }
                else if (payload.Command is ChoCalendarCommand)
                {
                    if (_workflowScheduler != null)
                        _workflowScheduler.ResetCalendar(payload.GetCommand<ChoCalendarCommand>().CalendarName);
                }
                else if (payload.Command is ChoSubscribeCommand)
                {
                    if (connState != null)
                    {
                        ChoSubscribeCommand cmd = payload.Command as ChoSubscribeCommand;
                        cmd.IPAddress = ((IPEndPoint)connState.RemoteEndPoint).Address.ToString();
                        if (_wsControllerConnectionManager != null)
                            _wsControllerConnectionManager.Subscribe(cmd, connState);
                    }
                }
                else if (payload.Command is ChoUsersACLChangedCommand)
                {
                    //if (_connectionManager != null)
                    //    _connectionManager.BroadcastCommand(payload);
                }
                else if (payload.Command is ChoWFDefsChangedCommand)
                    ChoWFDefsFolderSynchronizer.Value.SyncNow();
                else if (payload.Command is ChoWFDataChangedCommand)
                    ChoWFDataFolderSynchronizer.Value.SyncNow();
                else if (payload.Command is ChoWFPlugInsChangedCommand)
                    ChoWFPlugInsFolderSynchronizer.Value.SyncNow();
            }
            catch (Exception ex)
            {
                ChoTrace.Error(ex);
            }
        }

        private void ProcessTaskMgmtCommand(ChoTaskMgmtCommand taskMgmtCommand, DateTime timeStamp, string userName, string machineName)
        {
            if (!taskMgmtCommand.TaskGroupName.IsNullOrWhiteSpace() && taskMgmtCommand.TaskName.IsNullOrWhiteSpace())
                UpdateTaskGroup(taskMgmtCommand, userName);
            else if (!taskMgmtCommand.TaskName.IsNullOrWhiteSpace())
                UpdateTask(taskMgmtCommand, userName, machineName);
        }

        private void UpdateTaskGroup(ChoTaskMgmtCommand taskMgmtCommand, string userName)
        {
            if (taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Add
                || taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Update)
            {
                //Do nothing
            }
            else if (taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Remove)
            {
                if (_workflowScheduler != null)
                {
                    ChoTasksDataSet.CHO_TASKSDataTable tbl = ChoTaskModel.GetData(taskMgmtCommand.TaskGroupName, null);
                    foreach (ChoTasksDataSet.CHO_TASKSRow row in tbl.Rows)
                    {
                        _workflowScheduler.RemoveJobFromScheduler(row.TASK_ID, row.TASK_NAME, row.SCHEDULE_NAME);
                    }
                }
                //Do nothing
            }
        }

        private void UpdateTask(ChoTaskMgmtCommand taskMgmtCommand, string userName, string machineName)
        {
            string errMsg = null;
            string msg = null;
            if (taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Add)
            {
                ChoETLWorkflowStatus status = GetTaskStatus(taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName);

                //Add the task in Schedular
                if (_workflowScheduler != null)
                {
                    DateTimeOffset? nextRuntime = _workflowScheduler.UpdateJobToScheduler(taskMgmtCommand.TaskId, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, taskMgmtCommand.WorkflowName, status, out errMsg, out msg);
                    SetTaskStatus(0, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, ChoETLWorkflowStatus.INACTIVE, userName, machineName, String.Empty,
                        msg.IsNullOrWhiteSpace() ? _workflowScheduler.FormatNextRunTimeMessage(nextRuntime, errMsg) : msg);
                }
            }
            else if (taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Update)
            {
                ChoETLWorkflowStatus status = GetTaskStatus(taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName);
                //Update the task in Schedular
                ChoTaskGroupCategory taskGroupCategory = new ChoTaskGroupCategory(0, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, null);
                ChoTaskModel.Refresh(taskGroupCategory);
                DateTimeOffset? nextRuntime = _workflowScheduler.UpdateJobToScheduler(taskGroupCategory.Id, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, taskMgmtCommand.WorkflowName, status, out errMsg, out msg);
                SetTaskStatus(taskGroupCategory.BatchId, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, (ChoETLWorkflowStatus)taskGroupCategory.TaskStatus, userName, machineName, String.Empty,
                    msg.IsNullOrWhiteSpace() ? _workflowScheduler.FormatNextRunTimeMessage(nextRuntime, errMsg) : msg);
            }
            else if (taskMgmtCommand.TaskMgmtType == ChoTaskChangeType.Remove)
            {
                //Remove the task from Schedular
                ChoTaskGroupCategory taskGroupCategory = new ChoTaskGroupCategory(0, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, null);
                ChoTaskModel.Refresh(taskGroupCategory);
                //SetTaskStatus(taskGroupCategory.BatchId, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName, (ChoETLWorkflowStatus)taskGroupCategory.TaskStatus, userName, machineName, String.Empty, null);
                if (_workflowScheduler != null)
                    _workflowScheduler.RemoveJobFromScheduler(taskGroupCategory.Id, taskMgmtCommand.TaskName, taskMgmtCommand.ScheduleName);
            }
        }

        private void ProcessTaskIPCCommand(ChoTaskActionCommand taskCommand, DateTime timeStamp, string userName)
        {
            try
            {
                if (taskCommand.TaskName != _wfExecutionLineArgsObject.TaskName)
                    return;

                if (_lastModifiedDate >= timeStamp)
                    return;

                if (GetTaskMachineName(taskCommand.TaskName, taskCommand.ScheduleName).Item2 != _machineName)
                    return;

                if (taskCommand.CommandType == ChoTaskActionType.StopTask)
                {
                    if (_runningWF != null && _runningWF.BatchId == taskCommand.BatchId)
                    {
                        _runningWF.Tag = userName;
                        _runningWF.Stop();
                    }
                }
                else if (taskCommand.CommandType == ChoTaskActionType.AbortTask)
                {
                    if (_runningWF != null && _runningWF.BatchId == taskCommand.BatchId)
                    {
                        _runningWF.Tag = userName;
                        _runningWF.Abort();
                    }
                }
            }
            catch (Exception ex)
            {
                ChoTrace.Error(ex);
            }
        }

        [ChoStreamProfile("Failed to perform task commands...", Name = "wfFailedTasks", OuterProfileName = null)]
        private void ProcessTaskActionCommand(object state)
        {
            ChoWFServerCmdLineArgsObject wfServerCmdLineArgsObject = new ChoWFServerCmdLineArgsObject();
            ChoConnectionState conn = null;
            ChoCommandMessage payload = null;

            if (state is Tuple<object, ChoConnectionState>)
            {
                payload = ((Tuple<object, ChoConnectionState>)state).Item1 as ChoCommandMessage;
                conn = ((Tuple<object, ChoConnectionState>)state).Item2 as ChoConnectionState;
            }
            else
                payload = state as ChoCommandMessage;

            if (payload == null)
            {
                //ChoProfile.WriteLine("Missing payload.");
                return;
            }

            ChoTaskActionCommand command = payload.Command as ChoTaskActionCommand;
            if (command == null)
            {
                //ChoProfile.WriteLine("Missing command.");
                return;
            }

            lock (_taskActionCommandLock)
            {
                string machineName = null;
                try
                {
                    Tuple<string, string> machinePair = GetTaskMachineName(command.TaskName, command.ScheduleName);
                    machineName = machinePair.Item2;

                    if (!wfServerCmdLineArgsObject.RunAsPeer)
                    {
                        ChoTaskGroupCategory cat = GetTask(command.TaskName, command.ScheduleName);
                        ChoETLWorkflowStatus status = cat.WorkflowStatus; // _dataModel.GetTaskStatus(command.TaskName);

                        if (command.CommandType == ChoTaskActionType.ActivateTask)
                        {
                            machineName = ChoClusterManager.Default.Resolve(machinePair.Item1);

                            if (status == ChoETLWorkflowStatus.RUNNING)
                                return;

                            if (cat.BatchId != command.BatchId)
                            {
                                string errMsg = null;
                                string msg = null;
                                DateTimeOffset? nextRuntime = _workflowScheduler.UpdateJobToScheduler(cat.Id, cat.Name, cat.ScheduleName, cat.WorkflowName, ChoETLWorkflowStatus.ACTIVE, out errMsg, out msg);
                                SetTaskStatus(command.BatchId, command.TaskName, command.ScheduleName, errMsg.IsNullOrWhiteSpace() ? ChoETLWorkflowStatus.ACTIVE : ChoETLWorkflowStatus.INACTIVE, payload.User, payload.OriginHostName, String.Empty,
                                    msg.IsNullOrWhiteSpace() ? _workflowScheduler.FormatNextRunTimeMessage(nextRuntime, errMsg) : msg, machineName);
                            }

                            return;
                        }
                        else if (command.CommandType == ChoTaskActionType.InactivateTask)
                        {
                            if (status == ChoETLWorkflowStatus.RUNNING)
                                return;

                            //if (cat.BatchId == command.BatchId)
                            SetTaskStatus(cat.BatchId, command.TaskName, command.ScheduleName, ChoETLWorkflowStatus.INACTIVE, payload.User, payload.OriginHostName, String.Empty, "Task is NOT active.", String.Empty);
                            _workflowScheduler.RemoveJobFromScheduler(cat.Id, cat.Name, cat.ScheduleName);
                            return;
                        }

                        if (GetTaskStatus(command.TaskName, command.ScheduleName) == ChoETLWorkflowStatus.INACTIVE)
                        {
                            SetTaskStatus(cat.BatchId, command.TaskName, command.ScheduleName, ChoETLWorkflowStatus.INACTIVE, payload.User, payload.OriginHostName, String.Empty, "Task is NOT active.", String.Empty);
                            return;
                        }

                        if (command.CommandType == ChoTaskActionType.StopTask)
                        {
                            _workflowScheduler.CancelDependentTask(command.TaskName, command.ScheduleName);
                        }
                        else if (command.CommandType == ChoTaskActionType.AbortTask)
                        {
                            _workflowScheduler.CancelDependentTask(command.TaskName, command.ScheduleName);
                        }

                        //if (machineName.IsNullOrWhiteSpace())
                        //    throw new ApplicationException("Can't perform '{0}' task command. No host '{1}' defined for this task.".FormatString(command.TaskName, machineName));

                        if (machineName != _machineName)
                        {
                            if (IsPeerServerExists(machineName))
                            {
                                if (_wsControllerConnectionManager != null)
                                {
                                    //_wsControllerConnectionManager.BroadcastCommand(payload);
                                    _wsControllerConnectionManager.SendCommandToServer(machineName, payload);
                                    return;
                                }
                            }
                            else
                            {
                                if (command.CommandType == ChoTaskActionType.AbortTask)
                                {
                                    _workflowScheduler.CancelDependentTask(command.TaskName, command.ScheduleName);
                                    Task.Factory.StartNew(() => AbortTask(payload)).LogExceptions();
                                    return;
                                }
                                else
                                {
                                    throw new ApplicationException("Can't perform '{0}' task command. No host '{1}' found.".FormatString(command.TaskName, machineName));
                                }
                            }
                        }
                    }

                    if (machineName == Environment.MachineName)
                    {
                        if (command.CommandType == ChoTaskActionType.StartTask
                        || command.CommandType == ChoTaskActionType.ResumeTask)
                        {
                            ChoTaskGroupCategory cat = GetTask(command.TaskName, command.ScheduleName);
                            ChoETLWorkflowStatus status = cat.WorkflowStatus; // _dataModel.GetTaskStatus(command.TaskName);

                            if (status == ChoETLWorkflowStatus.RUNNING)
                            {
                                ChoProfile.WriteLine("Can't start '{0}' task. Existing BatchId: {1}, New BatchId: {2}, Status: {3}".FormatString(
                                    command.TaskName, cat.BatchId, command.BatchId, status));
                                //SetTaskStatus(cat.BatchId, command.TaskName, command.ScheduleName, status, payload.User, payload.OriginHostName, String.Empty, "Can't start '{0}' task. Existing BatchId: {1}, New BatchId: {2}, Status: {3}".FormatString(
                                //    command.TaskName, cat.BatchId, command.BatchId, status), String.Empty);
                                return;
                            }

                            if (command.CommandType == ChoTaskActionType.StartTask && status == ChoETLWorkflowStatus.ACTIVE)
                            //&& (cat.User == ChoUser.SYSTEM_ACCT
                            //|| (cat.User != ChoUser.SYSTEM_ACCT && cat.BatchId != command.BatchId))
                            /* && (cat.BatchId == command.BatchId ||
                                (cat.BatchId != command.BatchId && cat.User == ChoUser.SYSTEM_ACCT))*///)
                            {
                                WriteDataToIPC(command.UniqueName, ChoCommandMessage.Default);

                                //Start executing task in new shell
                                string workflowFilePath = ChoWorkflowDefManager.GetWorkflowDefFilePath(command.WFName);
                                Task.Factory.StartNew(() => StartTask(payload, workflowFilePath)).LogExceptions();
                            }
                            else if (command.CommandType == ChoTaskActionType.ResumeTask && status != ChoETLWorkflowStatus.ACTIVE)
                            {
                                WriteDataToIPC(command.UniqueName, ChoCommandMessage.Default);

                                //Start executing task in new shell
                                string workflowFilePath = ChoWorkflowDefManager.GetWorkflowDefFilePath(command.WFName);
                                Task.Factory.StartNew(() => StartTask(payload, workflowFilePath)).LogExceptions();
                            }
                            else
                            {
                                ChoProfile.WriteLine("Can't start '{0}' task. Existing BatchId: {1}, New BatchId: {2}, Status: {3}".FormatString(
                                    command.TaskName, cat.BatchId, command.BatchId, status));
                                SetTaskStatus(cat.BatchId, command.TaskName, command.ScheduleName, status, payload.User, payload.OriginHostName, String.Empty, "Can't start '{0}' task. Existing BatchId: {1}, New BatchId: {2}, Status: {3}".FormatString(
                                    command.TaskName, cat.BatchId, command.BatchId, status), null);
                            }
                        }
                        else if (command.CommandType == ChoTaskActionType.StopTask)
                        {
                            //_workflowScheduler.CancelDependentTask(command.TaskName, command.ScheduleName);
                            Task.Factory.StartNew(() => StopTask(payload)).LogExceptions();
                        }
                        else if (command.CommandType == ChoTaskActionType.AbortTask)
                        {
                            //_workflowScheduler.CancelDependentTask(command.TaskName, command.ScheduleName);
                            Task.Factory.StartNew(() => AbortTask(payload)).LogExceptions();
                        }
                    }
                }
                catch (Exception ex)
                {
                    ChoTrace.Error(ex);
                    if (!machineName.IsNullOrWhiteSpace())
                    {
                        ChoCommandMessage commandMessage = ChoCommandMessage.New(payload.OriginHostName, payload.User);
                        commandMessage.Command = new ChoServerMsgCommand() { Message = ex.Message, Detail = ex.ToString(), Severity = Severity.Error };

                        _wsControllerConnectionManager.SendTo(conn, commandMessage);
                    }
                }
            }
        }

        private bool IsPeerServerExists(string machineName)
        {
            if (_wsControllerConnectionManager == null) return false;

            return !_wsControllerConnectionManager.GetServerConnectionState(machineName).IsNullOrEmpty();
        }

        private void WriteDataToIPC(string name, ChoCommandMessage payload)
        {
            if (name.IsNullOrWhiteSpace()) return;

            lock (_mmwLock)
            {
                if (!_mmwDict.ContainsKey(name))
                    _mmwDict.Add(name, new ChoMemoryMappedFileWriter<ChoCommandMessage>(ChoMemoryMappedFileHelper.GetName(name)));

                _mmwDict[name].WriteData(payload);
            }
        }

        private void RemoveIPC(string name)
        {
            if (name.IsNullOrWhiteSpace()) return;

            lock (_mmwLock)
            {
                if (!_mmwDict.ContainsKey(name))
                    return;

                _mmwDict.Remove(name);
            }
        }

        private void StopTask(ChoCommandMessage payload)
        {
            ChoTaskActionCommand command = payload.Command as ChoTaskActionCommand;
            if (command == null || command.TaskName.IsNullOrWhiteSpace()) return;

            payload.TimeStamp = DateTime.Now;
            WriteDataToIPC(command.UniqueName, payload);
        }

        private void AbortTask(ChoCommandMessage payload)
        {
            ChoTaskActionCommand command = payload.Command as ChoTaskActionCommand;
            if (command == null || command.TaskName.IsNullOrWhiteSpace()) return;

            payload.TimeStamp = DateTime.Now;
            WriteDataToIPC(command.UniqueName, payload);
            Thread.Sleep(1000);
            try
            {
                SetTaskStatus(command.BatchId, command.TaskName, command.ScheduleName, ChoETLWorkflowStatus.ABORTED, payload.User, payload.OriginHostName);
                ChoTaskProcessManager.KillTask(command.TaskName);
            }
            catch (Exception ex)
            {
                ChoTrace.Error(ex);
            }
        }

        private void StartTask(ChoCommandMessage payload, string workflowFilePath)
        {
            ChoTaskActionCommand taskActionCmd = payload.Command as ChoTaskActionCommand;
            if (taskActionCmd == null || taskActionCmd.TaskName.IsNullOrWhiteSpace()) return;

            string taskName = taskActionCmd.TaskName;
            string scheduleName = taskActionCmd.ScheduleName;
            string cmdParameters = taskActionCmd.CommandParameters;
            int batchId = taskActionCmd.BatchId;
            string userName = payload.User;
            string modifiedByHostName = payload.OriginHostName;
            bool isResumeTask = taskActionCmd.CommandType == ChoTaskActionType.ResumeTask;

            if (!isResumeTask)
                ChoTrace.WriteLine("Starting '{0}' task...".FormatString(taskName));
            else
                ChoTrace.WriteLine("Resuming '{0}' task...".FormatString(taskName));

            try
            {
                //Kill any arphan process for this task
                ChoTaskProcessManager.KillTask(taskName);

                _concurrencySemaphore.Wait();

                if (workflowFilePath.IsNullOrWhiteSpace())
                    throw new ApplicationException("Can't find workflow definition for {0} task.".FormatString(taskName));
                if (userName.IsNullOrWhiteSpace())
                    throw new ApplicationException("Missing user information initiated the request.");

                // Use ProcessStartInfo class
                ProcessStartInfo startInfo = new ProcessStartInfo();
                startInfo.CreateNoWindow = _workflowServerSettings.CreateNoWindow;
                startInfo.UseShellExecute = false;
                startInfo.FileName = _worklowServiceProgramFilePath;
                startInfo.WorkingDirectory = Path.GetDirectoryName(_worklowServiceProgramFilePath);
                //startInfo.WindowStyle = ProcessWindowStyle.Hidden;

                startInfo.Arguments = "RUN_WF /t:\"{0}\" /s:\"{1}\" /b:\"{2}\" /w:\"{3}\" /u:\"{4}\" {5} /p:\"{6}\" /to:{7} /sr:{8} /mr:{9} /d-".FormatString(
                    taskName, scheduleName, batchId, workflowFilePath, userName,
                    isResumeTask ? "/r" : String.Empty, ChoString.ExpandPropertiesEx(cmdParameters),
                    ChoTaskTimeout.Resolve(taskActionCmd.Timeout) * 1000, taskActionCmd.SleepBetweenRetry * 1000, taskActionCmd.MaxNoOfRetry);

                if (_dataModel.GetTaskStatus(taskName, scheduleName) == ChoETLWorkflowStatus.RUNNING)
                {
                    ChoTrace.Info("{0}: Task is in running state.".FormatString(taskName));
                    return;
                }

                ChoTrace.WriteLine("Lanching workflow with below arguments...");
                ChoTrace.WriteLine(startInfo.Arguments);

                SetTaskStatus(batchId, taskName, scheduleName, ChoETLWorkflowStatus.ACTIVE, userName, modifiedByHostName, String.Empty);
                //SetTaskStatus(batchId, taskName, scheduleName, ChoETLWorkflowStatus.RUNNING, userName, modifiedByHostName, String.Empty);

                // Start the process with the info we specified.
                // Call WaitForExit and then the using statement will close.

                int counter = 0;
                while (true)
                {
                    counter++;

                    using (Process exeProcess = Process.Start(startInfo))
                    {
                        if (exeProcess == null)
                            throw new ApplicationException("Can't start workflow process.");

                        try
                        {
                            ChoTaskProcessManager.Register(taskName, exeProcess);

                            ChoTrace.WriteLine("task '{0}', ProcessId: {1}...".FormatString(taskName, exeProcess.Id));

                            exeProcess.WaitForExit();
                            ChoTrace.WriteLine("task '{0}', ProcessId: {1} - Completed...".FormatString(taskName, exeProcess.Id));
                            Thread.Sleep(_workflowServerSettings.SoftAbortTimeout * 2);

                            ChoETLWorkflowStatus status = _dataModel.GetTaskStatus(taskName, scheduleName);
                            if (status == ChoETLWorkflowStatus.ACTIVE)
                            {
                                //if (counter >= 3)
                                throw new ApplicationException("Catastrophic failure. Failed to run process.");
                                //else
                                //    continue;
                            }
                            else if (status == ChoETLWorkflowStatus.RUNNING)
                            {
                                taskActionCmd.CommandType = ChoTaskActionType.StopTask;
                                StopTask(payload);

                                Thread.Sleep(_workflowServerSettings.SoftAbortTimeout * 2);
                                status = _dataModel.GetTaskStatus(taskName, scheduleName);
                                if (status == ChoETLWorkflowStatus.RUNNING)
                                    throw new ApplicationException("Process terminated abnormally.");
                            }
                            else
                            {
                                if (exeProcess.ExitCode != 0)
                                {
                                    ChoTrace.WriteLine("{0}: Workflow [{1}] exited with exit code [{2}].".FormatString(taskName, workflowFilePath, exeProcess.ExitCode));
                                    //SetTaskStatus(batchId, taskName, scheduleName, ChoETLWorkflowStatus.FAILED, userName, modifiedByHostName, "{0}: Workflow [{1}] exited with exit code [{2}].".FormatString(taskName, workflowFilePath, exeProcess.ExitCode));
                                }
                                break;
                            }
                        }
                        finally
                        {
                            ChoTaskProcessManager.Unregister(taskName);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                // Log error.
                ChoTrace.Error("{0}: Failed to start workflow [{1}]. {2}".FormatString(taskName, workflowFilePath, ex.Message));
                SetTaskStatus(batchId, taskName, scheduleName, ChoETLWorkflowStatus.FAILED, userName, modifiedByHostName, ex.Message);
            }
            finally
            {
                _concurrencySemaphore.Release();
            }

            RemoveIPC(taskActionCmd.UniqueName);
            if (taskActionCmd.RunDependentTasks)
                _dataModel.FireDependentTasks(batchId, taskName, scheduleName, payload.User, payload.HostName);
        }
    }
}
